from anole  import AnoleLog
from anole.utils.URL import URL,URLError
from anole.utils.HTMLParser import BaseHTMLParser,html_tidy
from anole.core.processor import ContentP
from anole.contrib.other.simpleurlseen import UrlSeen 
#from anole.utils.urlfilter import URLFilter

class HTMLProcessor(ContentP):
    _CONTENT_TYPES=['text/html',]
    def __init__(self,pm):

		#init work in pull manner
        	self._usp=pm.get_usp()
        	self._config=pm.get_config()
       		self._log=AnoleLog().log
		self._uSeen=UrlSeen()

    def process_text_html(self,replyHead,contentFile,request):
        self._log('process '+request.strURL+'-->start ',AnoleLog.LEVEL_DEBUG)
	U=request.urlObj.__class__
        file=contentFile
        parser=BaseHTMLParser()
        #file.reset()   
        strHTML=file.read()
	self._log('LinkExtractor read html finish',AnoleLog.LEVEL_DEBUG)
       
        if not strHTML:
	    self._log('empty html',AnoleLog.LEVEL_WARNING)
	    self._log('process '+request.strURL+'-->succeed',AnoleLog.LEVEL_DEBUG)
            return
      
        #tidy the html
        strHTML=html_tidy(strHTML)

        try:
            parser.feed(strHTML)
        except:
            import traceback,sys
            traceback.print_exc(file=sys.stdout)
            self._log('parse html error',AnoleLog.LEVEL_ERROR)   
	    self._log('process '+request.strURL+'-->succeed',AnoleLog.LEVEL_DEBUG)
            return
       
        links=parser.links()
        for link in links:
            
            if not link:
                continue
            try:
                url=URL(link,baseURL=request.strURL)
                strFullURL=url.full_url(withFrag=False)
               
            except URLError,e:
                self._log(repr(e)+str(e),AnoleLog.LEVEL_WARNING)
                continue    
            else:
		if not self._uSeen.is_seen(strFullURL):		

			self._usp.put_url(U(strFullURL))
        self._log('process '+request.strURL+'-->succeed',AnoleLog.LEVEL_DEBUG)

        
    def _log(self,str,level=AnoleLog.LEVEL_DEBUG):
        self._logObj.log(str,level)
        pass
       
    
